From: Keir Fraser Date: Thu, 17 Dec 2009 06:27:56 +0000 (+0000) Subject: This patch establishes a new abstraction of sharing handles (encoded as a 64bit X-Git-Tag: archive/raspbian/4.8.0-1+rpi1~1^2~12860 X-Git-Url: https://dgit.raspbian.org/%22http:/www.example.com/cgi/%22https:/%22bookmarks://%22/%22http:/www.example.com/cgi/%22https:/%22bookmarks:/%22?a=commitdiff_plain;h=cf5e23f6c44689bcf1996cf890b1c61c6fdb89d0;p=xen.git This patch establishes a new abstraction of sharing handles (encoded as a 64bit int), each corresponding to a single sharable pages. Externally all sharing related operations (e.g. nominate/share) will use sharing handles, thus solving a lot of consistency problems (like: is this sharable page still the same sharable page as before). Internally, sharing handles can be translated to the MFNs (using a newly created hashtable), and then for each MFNs a doubly linked list of GFNs translating to this MFN is maintained. Finally, sharing handle is stored in page_info strucutre for each sharable MFN. All this allows to share and unshare pages efficiently. However, at the moment a single lock is used to protect the sharing handle hash table. For scalability reasons, the locking needs to be made more granular. Signed-off-by: Grzegorz Milos --- diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index 0a0df9ef9d..71cd08be1e 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -114,6 +114,7 @@ #include #include #include +#include /* * Mapping of first 2 or 4 megabytes of memory. This is mapped with 4kB @@ -314,6 +315,8 @@ void __init arch_init_memory(void) } subarch_init_memory(); + + mem_sharing_init(); } int page_is_ram_type(unsigned long mfn, unsigned long mem_type) diff --git a/xen/arch/x86/mm/mem_sharing.c b/xen/arch/x86/mm/mem_sharing.c index 064fb0d86b..0eee4efdeb 100644 --- a/xen/arch/x86/mm/mem_sharing.c +++ b/xen/arch/x86/mm/mem_sharing.c @@ -25,7 +25,11 @@ #include #include #include - +#include +#include +#include +#include + #undef mfn_to_page #define mfn_to_page(_m) __mfn_to_page(mfn_x(_m)) #undef mfn_valid @@ -33,6 +37,147 @@ #undef page_to_mfn #define page_to_mfn(_pg) _mfn(__page_to_mfn(_pg)) +static shr_handle_t next_handle = 1; + +typedef struct shr_hash_entry +{ + shr_handle_t handle; + mfn_t mfn; + struct shr_hash_entry *next; + struct list_head gfns; +} shr_hash_entry_t; + +#define SHR_HASH_LENGTH 1000 +static shr_hash_entry_t *shr_hash[SHR_HASH_LENGTH]; + +typedef struct gfn_info +{ + unsigned long gfn; + domid_t domain; + struct list_head list; +} gfn_info_t; + +typedef struct shr_lock +{ + spinlock_t lock; /* mem sharing lock */ + int locker; /* processor which holds the lock */ + const char *locker_function; /* func that took it */ +} shr_lock_t; +static shr_lock_t shr_lock; + +#define shr_lock_init(_i) \ + do { \ + spin_lock_init(&shr_lock.lock); \ + shr_lock.locker = -1; \ + shr_lock.locker_function = "nobody"; \ + } while (0) + +#define shr_locked_by_me(_i) \ + (current->processor == shr_lock.locker) + +#define shr_lock(_i) \ + do { \ + if ( unlikely(shr_lock.locker == current->processor) ) \ + { \ + printk("Error: shr lock held by %s\n", \ + shr_lock.locker_function); \ + BUG(); \ + } \ + spin_lock(&shr_lock.lock); \ + ASSERT(shr_lock.locker == -1); \ + shr_lock.locker = current->processor; \ + shr_lock.locker_function = __func__; \ + } while (0) + +#define shr_unlock(_i) \ + do { \ + ASSERT(shr_lock.locker == current->processor); \ + shr_lock.locker = -1; \ + shr_lock.locker_function = "nobody"; \ + spin_unlock(&shr_lock.lock); \ + } while (0) + + + +static void mem_sharing_hash_init(void) +{ + int i; + + shr_lock_init(); + for(i=0; ihandle == handle) + return e; + e = e->next; + } + + return NULL; +} + +static shr_hash_entry_t* mem_sharing_hash_insert(shr_handle_t handle, mfn_t mfn) +{ + shr_hash_entry_t *e, **ee; + + e = mem_sharing_hash_alloc(); + if(e == NULL) return NULL; + e->handle = handle; + e->mfn = mfn; + ee = &shr_hash[handle % SHR_HASH_LENGTH]; + e->next = *ee; + *ee = e; + return e; +} + +static void mem_sharing_hash_delete(shr_handle_t handle) +{ + shr_hash_entry_t **pprev, *e; + + pprev = &shr_hash[handle % SHR_HASH_LENGTH]; + e = *pprev; + while(e != NULL) + { + if(e->handle == handle) + { + *pprev = e->next; + mem_sharing_hash_destroy(e); + return; + } + pprev = &e->next; + e = e->next; + } + printk("Could not find shr entry for handle %lx\n", handle); + BUG(); +} + static struct page_info* mem_sharing_alloc_page(struct domain *d, unsigned long gfn, @@ -181,12 +326,18 @@ int mem_sharing_debug_gref(struct domain *d, grant_ref_t ref) int mem_sharing_nominate_page(struct domain *d, unsigned long gfn, - int expected_refcnt) + int expected_refcnt, + shr_handle_t *phandle) { p2m_type_t p2mt; mfn_t mfn; struct page_info *page; int ret; + shr_handle_t handle; + shr_hash_entry_t *hash_entry; + struct gfn_info *gfn_info; + + *phandle = 0UL; mfn = gfn_to_mfn(d, gfn, &p2mt); @@ -205,6 +356,22 @@ int mem_sharing_nominate_page(struct domain *d, if(ret) goto out; + /* Create the handle */ + ret = -ENOMEM; + shr_lock(); + handle = next_handle++; + if((hash_entry = mem_sharing_hash_insert(handle, mfn)) == NULL) + { + shr_unlock(); + goto out; + } + if((gfn_info = mem_sharing_gfn_alloc()) == NULL) + { + mem_sharing_hash_destroy(hash_entry); + shr_unlock(); + goto out; + } + /* Change the p2m type */ if(p2m_change_type(d, gfn, p2mt, p2m_ram_shared) != p2mt) { @@ -213,18 +380,77 @@ int mem_sharing_nominate_page(struct domain *d, * The mfn needs to revert back to rw type. This should never fail, * since no-one knew that the mfn was temporarily sharable */ ASSERT(page_make_private(d, page) == 0); + mem_sharing_hash_destroy(hash_entry); + mem_sharing_gfn_destroy(gfn_info); + shr_unlock(); goto out; } /* Update m2p entry to SHARED_M2P_ENTRY */ set_gpfn_from_mfn(mfn_x(mfn), SHARED_M2P_ENTRY); + INIT_LIST_HEAD(&hash_entry->gfns); + INIT_LIST_HEAD(&gfn_info->list); + list_add(&gfn_info->list, &hash_entry->gfns); + gfn_info->gfn = gfn; + gfn_info->domain = d->domain_id; + page->shr_handle = handle; + *phandle = handle; + shr_unlock(); + ret = 0; out: return ret; } +int mem_sharing_share_pages(shr_handle_t sh, shr_handle_t ch) +{ + shr_hash_entry_t *se, *ce; + struct page_info *spage, *cpage; + struct list_head *le, *te; + struct gfn_info *gfn; + struct domain *d; + int ret; + + shr_lock(); + + ret = -1; + se = mem_sharing_hash_lookup(sh); + if(se == NULL) goto err_out; + ret = -2; + ce = mem_sharing_hash_lookup(ch); + if(ce == NULL) goto err_out; + spage = mfn_to_page(se->mfn); + cpage = mfn_to_page(ce->mfn); + list_for_each_safe(le, te, &ce->gfns) + { + gfn = list_entry(le, struct gfn_info, list); + /* Get the source page and type, this should never fail + * because we are under shr lock, and got non-null se */ + BUG_ON(!get_page_and_type(spage, dom_cow, PGT_shared_page)); + /* Move the gfn_info from ce list to se list */ + list_del(&gfn->list); + d = get_domain_by_id(gfn->domain); + BUG_ON(!d); + BUG_ON(set_shared_p2m_entry(d, gfn->gfn, se->mfn) == 0); + put_domain(d); + list_add(&gfn->list, &se->gfns); + put_page_and_type(cpage); + } + ASSERT(list_empty(&ce->gfns)); + mem_sharing_hash_delete(ch); + /* Free the client page */ + if(test_and_clear_bit(_PGC_allocated, &cpage->count_info)) + put_page(cpage); + ret = 0; + +err_out: + shr_unlock(); + + return ret; +} + int mem_sharing_unshare_page(struct domain *d, unsigned long gfn, uint16_t flags) @@ -233,19 +459,63 @@ int mem_sharing_unshare_page(struct domain *d, mfn_t mfn; struct page_info *page, *old_page; void *s, *t; - int ret; + int ret, last_gfn; + shr_hash_entry_t *hash_entry; + struct gfn_info *gfn_info = NULL; + shr_handle_t handle; + struct list_head *le; mfn = gfn_to_mfn(d, gfn, &p2mt); page = mfn_to_page(mfn); - + handle = page->shr_handle; + + /* Remove the gfn_info from the list */ + shr_lock(); + hash_entry = mem_sharing_hash_lookup(handle); + list_for_each(le, &hash_entry->gfns) + { + gfn_info = list_entry(le, struct gfn_info, list); + if((gfn_info->gfn == gfn) && (gfn_info->domain == d->domain_id)) + goto gfn_found; + } + printk("Could not find gfn_info for shared gfn: %lx\n", gfn); + BUG(); +gfn_found: + /* Delete gfn_info from the list, but hold on to it, until we've allocated + * memory to make a copy */ + list_del(&gfn_info->list); + last_gfn = list_empty(&hash_entry->gfns); + + /* If the GFN is getting destroyed drop the references to MFN + * (possibly freeing the page), and exit early */ + if(flags & MEM_SHARING_DESTROY_GFN) + { + mem_sharing_gfn_destroy(gfn_info); + if(last_gfn) mem_sharing_hash_delete(handle); + shr_unlock(); + put_page_and_type(page); + if(last_gfn && + test_and_clear_bit(_PGC_allocated, &page->count_info)) + put_page(page); + return 0; + } + ret = page_make_private(d, page); + BUG_ON(last_gfn & ret); if(ret == 0) goto private_page_found; old_page = page; page = mem_sharing_alloc_page(d, gfn, flags & MEM_SHARING_MUST_SUCCEED); BUG_ON(!page && (flags & MEM_SHARING_MUST_SUCCEED)); - if(!page) return -ENOMEM; + if(!page) + { + /* We've failed to obtain memory for private page. Need to re-add the + * gfn_info to relevant list */ + list_add(&gfn_info->list, &hash_entry->gfns); + shr_unlock(); + return -ENOMEM; + } s = map_domain_page(__page_to_mfn(old_page)); t = map_domain_page(__page_to_mfn(page)); @@ -257,6 +527,11 @@ int mem_sharing_unshare_page(struct domain *d, put_page_and_type(old_page); private_page_found: + /* We've got a private page, we can commit the gfn destruction */ + mem_sharing_gfn_destroy(gfn_info); + if(last_gfn) mem_sharing_hash_delete(handle); + shr_unlock(); + if(p2m_change_type(d, gfn, p2m_ram_shared, p2m_ram_rw) != p2m_ram_shared) { @@ -269,5 +544,9 @@ private_page_found: return 0; } - +void mem_sharing_init(void) +{ + printk("Initing memory sharing.\n"); + mem_sharing_hash_init(); +} diff --git a/xen/include/asm-x86/mem_sharing.h b/xen/include/asm-x86/mem_sharing.h index 3b4f4fea77..60c74b98c5 100644 --- a/xen/include/asm-x86/mem_sharing.h +++ b/xen/include/asm-x86/mem_sharing.h @@ -24,9 +24,13 @@ #define sharing_supported(_d) \ (is_hvm_domain(_d) && (_d)->arch.hvm_domain.hap_enabled) + +typedef uint64_t shr_handle_t; + int mem_sharing_nominate_page(struct domain *d, unsigned long gfn, - int expected_refcnt); + int expected_refcnt, + shr_handle_t *phandle); #define MEM_SHARING_MUST_SUCCEED (1<<0) #define MEM_SHARING_DESTROY_GFN (1<<1) int mem_sharing_unshare_page(struct domain *d, @@ -34,5 +38,6 @@ int mem_sharing_unshare_page(struct domain *d, uint16_t flags); int mem_sharing_sharing_resume(struct domain *d); int mem_sharing_cache_resize(struct domain *d, int new_size); +void mem_sharing_init(void); #endif /* __MEM_SHARING_H__ */ diff --git a/xen/include/asm-x86/mm.h b/xen/include/asm-x86/mm.h index 5c3ab88c3a..5f8aeb5d00 100644 --- a/xen/include/asm-x86/mm.h +++ b/xen/include/asm-x86/mm.h @@ -43,6 +43,8 @@ struct page_info struct page_list_entry list; /* For non-pinnable shadows, a higher entry that points at us. */ paddr_t up; + /* For shared/sharable pages the sharing handle */ + uint64_t shr_handle; }; /* Reference count and various PGC_xxx flags and fields. */